package com.ibilibili.crawl;

import org.apache.http.client.HttpClient;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.util.EntityUtils;
import org.apache.http.impl.client.HttpClients;

public class CrawlMain {
    private static final int tidTotal = 1000;
    private static final int pnTotal = 100000;
    public static void main(String[] args) {
        run();
    }

    public static void run() {
        int tid = 0;
        while (true) {
            if (tid == tidTotal) {
                break;
            }
            try{
                for (int pn = 0; pn <= pnTotal; pn++) {
                    HttpClient httpClient = HttpClients.createDefault();
                    String requestUrl = "http://api.bilibili.com/archive_rank/getarchiverankbypartion?callback=?&type=json&tid=" + tid + "&pn=" + pn;
                    HttpGet req = new HttpGet(requestUrl);
                    req.addHeader("Accept", "application/json, text/javascript, */*; q=0.01");
                    req.addHeader("Accept-Encoding", "gzip,deflate,sdch");
                    req.addHeader("Accept-Language", "zh-CN,zh;q=0.8");
                    req.addHeader("Content-Type", "textml; charset=UTF-8");
                    req.addHeader("User-Agent", "Mozilla/5.0 (Windows NT 5.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36");
                    HttpResponse rep = httpClient.execute(req);
                    HttpEntity repEntity = rep.getEntity();
                    String jsonMessage = EntityUtils.toString(repEntity);
                    JSONDeal jd = new JSONDeal(jsonMessage);
                    jd.parse();
                    Thread.sleep(10);
                    System.out.println("tid: " + tid + ", pn: " + pn);
                }

            } catch(Exception e){
                e.printStackTrace();
                try {
                    System.out.println("crawl exception sleep 10s continue...");
                    Thread.sleep(10000);
                }catch (Exception ee) {}
            }
            tid ++;
        }
    }
}
